######## Classifier Data Script ########
######## Final Year Project 2011#####
######## Making Web Search Spatial###


######## Liam Sullivan       ########
######## University of Leeds ########

######## Script to pull back classification data for 100 search queries, 
######## each having 100 search results in total

import psyco
psyco.full()


#### Due to Django / Google APp Engine problems need to copy all code rather than importing from Django files

import csv
import urllib
import StringIO


Problems = False


import urllib
import logging
import time


from django import *
from operator import itemgetter
from StringIO import StringIO


class SearchResult():
    
    """SearchResult objects have a Title, Extract, URL and Classification data"""
    
    def __init__(self, URL, ClickURL, Title, Extract):
        
        """Build object with passed in values but also generate Classification data from URL and encode in Object"""
        self.URL = URL
        self.ClickURL = ClickURL
        self.Title = Title
        self.Extract = Extract
               
        #Get classification data
        self.Classification = getClassificationData(URL)
        
        #Multiply classification values by 100 to get probabilities as %
        for key, value in self.Classification.iteritems():
            
            ####logging.info("It was: ")
            ####logging.info(value)
            NewValue = value * 100
            
            ####logging.info("Its now: ")
            ####logging.info(NewValue)
            
            NewValue = int(NewValue)
            
            ####logging.info("Finally after rounding it is: ")
            ####logging.info(NewValue)
            
            
            #And alter the value for the key
            self.Classification[key] = NewValue
            
        self.Arts = self.Classification['Arts']
        self.Business = self.Classification['Business']
        self.Science = self.Classification['Science']
        self.Computers = self.Classification['Computers']
        self.Recreation = self.Classification['Recreation']
        self.Sports = self.Classification['Sports']
        self.Society = self.Classification['Society']
        self.Health = self.Classification['Health']
        self.Home = self.Classification['Home']  
        self.Games = self.Classification['Games'] 
        
 
def getSearchResults(appid, query, results, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = ''):
    
    """Get search results, call out to classify each one and build a SearchResult object for each, return a list of these"""
   
    base_url = u'http://search.yahooapis.com/WebSearchService/V1/webSearch?'
    params = locals()
    params['output'] = 'json'
    payload = urllib.urlencode(params)
    url = base_url + payload
    
    responseContent = urllib.urlopen(url)    
    response = StringIO(responseContent.read())


    result = simplejson.load(response)
    
    ProblemsHere = False
    
    try:
        result = result['ResultSet']
    except:
        ProblemsHere = True   
     
    try:
        AllResults = result['Result']
    except:
        ProblemsHere = True
        
        
    ResultObjectList = []
    
    resultCounter = 0
    
    FakedResults = False
    
    
    
    for res in AllResults:
        
        if not ProblemsHere:
            
            resultCounter +=1
            
            #Make a new SearchResult Object
            SearchResultInstance = SearchResult(res['Url'], res['ClickUrl'], res['Title'], res['Summary'])
            
            #Use the object method to fetch classification data before saving object
            
            #Add it to the list
            
            ResultObjectList.append(SearchResultInstance)
            print "Got result number: " + str(resultCounter)
            
            
            
        
        else:
            
            FakeResultObjectList = []
            
            FakedResults = True
            
            #Make a fake SearchResult Object
            FakeSearchResultInstance = SearchResult('http://www.google.com', 'http://www.google.com', '9879 Error Fake Data - Ignore Entry', 'A fake entry to avoid halting, ignore this entry as something went wrong')
            #Add it to the list
            ResultObjectList.append(FakeSearchResultInstance)
            
            
            
            
    if FakedResults:
        print "Error - Returned FAKE RESULTS"
        return FakeResultObjectList
    
    else:
        
        print "Returned ALL RESULTS FOR THAT QUERY"
        return ResultObjectList



def getClassificationData(URL):
    """Classifies a URL using the UClassify Service"""
    
    #Dictionary containing all params, which we will encode for REST Call    
    Params = {}   
        
    #BaseURL
    #uClassify URL Endpoint
    endpoint = "http://uclassify.com/browse/uClassify/Topics/ClassifyUrl?"
    
    #API Key    
    Params['readkey'] = "TsE3A35FCBSAdbGFLEvI2PXH0"
    
    #Output Format
    Params['output'] = "json"
    
    #Incoming URL that needs classifying
    Params['url'] = URL
    
    #Version
    Params['version'] = 1.01
    
    #Encode Params from dictionary for call
    
    RestParams = urllib.urlencode(Params)    
    
    #Combine base url and params
    
    RestURL = endpoint + RestParams
    
    #time.sleep(0.1)
        
    try:        
        responseContent = urllib.urlopen(RestURL)    
        ClassifierResponse = StringIO(responseContent.read())
        
    except:
        "Failed to get classification"
        
 
    ClassifierResult = simplejson.load(ClassifierResponse)
    
    ##Fake empty results incase don't get any
    
    ResultSet = {'Arts': 0, u'Business': 0, u'Science': 0, u'Computers': 0, u'Recreation': 0, u'Sports': 0, u'Society': 0, u'Health': 0, u'Home': 0, u'Games': 0}
    
    try:
        ResultSet =  ClassifierResult['cls1'] 
        return ResultSet 
    except:
        print "didn't get ANY CLASSIFICATION DATA so pass empty data {}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}"
        Problems = True
        return ResultSet
             
        

#Yahoo APP ID
appid = 'Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-'

### How many search results for each query??     
NumberOfResults = 100
###

#Read in Queries

QueryListFile = open("QueryListFile.txt")

QueryList = []

count = 0

for line in QueryListFile.readlines():
    
    QueryList.append(str(line))
    count += 1
    print "Read query number: " + str(count) + " which is: " + line
    
print "Read in " + str(count) + " queries to pull data for"


#Prepare output file
OutFile = csv.writer(open("ClassifierTestResults.csv", 'w'), dialect='excel')

#Write headings to file followed by new line

Headings = ['Query', 'Result Title','Click URL', 'URL', 'Arts','Games','Home', 'Health', 'Society', 'Sports', 'Recreation','Computers', 'Science']

OutFile.writerow(Headings)

#For each query use existing methods to build a SearchResult object as before

QueryCount = 0

for query in QueryList:
    
    QueryCount += 1
    
    print "Getting search results for query number: " + str(QueryCount)
        
    #Get a list of SearchResult objects
    ResultObjectsList = getSearchResults(appid, query, NumberOfResults, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = '')
    
    ResultCount = 0
    
    for res in ResultObjectsList:
                
        ResultCount ++1       
        
        #For each result output a line with the result details and all classification data
        
        OutLine = [query, res.Title, res.ClickURL, res.URL, res.Arts, res.Games, res.Home,  res.Health, res.Society, res.Sports, res.Recreation, res.Computers, res.Science]
        
        try:
            OutFile.writerow(OutLine)
        except:
            print "ERROR - Skipped a line as couldn't 'print it, seomthing wrong with characters {}{}{}{}{}{"
        
    print "Saved classification data for " + str(ResultCount) + " results from that query..."
    
    if Problems:
        print "But had at least one problem"


print "Done!"
    

